#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib2
import os, sys
import re
import time
from Queue import Queue
from threading import Thread, Lock, stack_size

import pwpd.core.pwdlogging as Logging
import pwpd.common.config as Config
import pwpd.common.charsetck as charsetck

HEAD_REG = re.compile(r'(<head>.*</head>)', re.DOTALL | re.IGNORECASE)
TITLE_REG = re.compile(r'<title>([^<]*)</title>', re.IGNORECASE)

logger = Logging.get_logger(__name__)
DEBUG = True

class CustomedRedirectHandler(urllib2.HTTPRedirectHandler):
    
    def http_error_302(self, req, fp, code, msg, headers):
        ret = urllib2.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers)
        
        ret.status = code
        return ret
    
    http_error_301 = http_error_302

class PhishingPageDetector:
    def __init__(self, config, ppfeature = None, timeout = 15):
        
        stack_size(32768*16)
        #
        self.config = config
        self.__load_config(config)
        self.ppfeature = ppfeature
        self.ppkw_list = ppfeature.get_ppkw_list()
        self.timeout = timeout
        #
        self.opener = urllib2.build_opener(urllib2.HTTPHandler)
        self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 (.NET CLR 3.5.30729)'),
                                  ('Accept-Language', 'zh-cn;en-us;q=0.7,en;q=0.3'),
#                                  ('Accept-Encoding', 'gzip, deflate'),
                                  ('Connection', 'keep-alive')]
        # set timeout.
        urllib2.socket.setdefaulttimeout(timeout)
        #
        self.lock = Lock()
        self.req_q = Queue()
        self.ans_q = Queue()
        #
        self._running = 0
        self.__init_fetcher(self.threads_w)
        
    def __del__(self):
        time.sleep(0.5)
        self.req_q.join()
        self.ans_q.join()                
    
    def __load_config(self, config):
        self.threads_w = int(config.get_opt(Config.OPT_WORK_NUM))
        self.threads_p = int(config.get_opt(Config.OPT_PRODUCER_NUM))
    
    def __init_fetcher(self, threads_num):
        
        for i in range(threads_num):
            t = Thread(target = self.__fetcher_thread)
            t.setDaemon(True)
            t.start()
    
    def __fetcher_thread(self):
        while True:
            req = self.req_q.get()
            with self.lock:
                self._running += 1

            # GET page, identify the HTTP code.
            fobj = self.__fetcher_page(req)
            # only handle the page which returned HTTP code is 200 and has not been redirected.
            #
            if fobj and fobj.code == 200 and not hasattr(fobj, 'status'):
                try:
                    page_cont = fobj.read()
                except Exception, e:
                    print >> sys.stderr, 'read PAGE CONTENT error: %s\t%s' %(str(e), fobj.geturl())
                    logger.debug('read PAGE CONTENT error: %s\t%s', str(e), fobj.geturl())
                else:
                    if self.__detect_page_head(page_cont):
                        self.ans_q.put((req, page_cont))
                        logger.info('**** PHISHING page detected, URL: %s', req)
                        #
                        if DEBUG:
                            fname = os.path.join(os.getcwd(), self.config.get_opt(Config.OPT_OUTPUT), \
                                                 req.split('//')[1].replace('/', '_') + '.html')
                            snap_fd = open(fname, 'w')
                            print >> snap_fd, page_cont
                            
                            snap_fd.close()
                finally:
                    #
                    with self.lock:
                        self._running -= 1
                    #
                    self.req_q.task_done()
                    time.sleep(0.1)
            else:
                with self.lock:
                    self._running -= 1
                self.req_q.task_done()
                time.sleep(0.1)
    #
    #@return: file-like object returned by urllib2.urlopen()
    def __fetcher_page(self, url):
        '''
        @note: Get the page content; if the page does not exist, empty return string.
        '''
        ret = None
        try:
#            cont = self.opener.open(url, timeout = self.timeout).read()
            ret = self.opener.open(url)
            logger.debug('target page EXIST %s:\t', url)
        except urllib2.HTTPError, e:
            logger.debug('catch HTTPError %s: \t %s', url, str(e.code))
        except urllib2.URLError, e:
            logger.debug('catch URLError %s: \t %s', url, str(e.reason))
        except Exception, e:
            logger.debug('catch Exception %s: \t %s', url, e)
        
        return ret
    
    def __detect_page_head(self, page):
        
        charset = charsetck.check(None, page)
        if not charset:
            charset = 'gb2312'
        
        is_phishing = False
        head_l = HEAD_REG.findall(page)
        ##
        ## @todo: more elegant and efficient, eliminate loop count.
        for ll in head_l:
            title_l = TITLE_REG.findall(ll)
            for lll in title_l:
                # detect whether contains key words.
                # pay attention to encoding.
                for kw in self.ppkw_list:
                    #
                    try:
                        lll_u = lll.decode(charset)

                    except Exception, e:
                        break
                    if re.search(kw, lll_u):
                        logger.info('phishing page title: %s', lll_u)
                        is_phishing = True
                        return is_phishing
        
        return is_phishing
    
# public methods.
    def push_task(self, req_url):
        self.req_q.put(req_url)
    
    def pop_pw(self):
        
        try:
            item = self.ans_q.get_nowait()
        except Exception, e:
            return None
        #
        self.ans_q.task_done()
        return item
#        return self.ans_q.get()
    
    def left_task(self):
        logger.debug('req_q.size = %d | ans_q.size = %d | running_threads = %d', 
                     self.req_q.qsize(), self.ans_q.qsize(), self._running)
        return self.req_q.qsize() + self.ans_q.qsize() + self._running
    
    def left_reqTask(self):
        return self.req_q.qsize()
        
class PhishingPageFeature:
    
    def __init__(self, config):
        self._kw_ll = list()
        self._pwsd_ll = list()
        self._pwsd_level = dict()
        self._raw_input_files = list()
        self._out_fname = ''
        
        self.__load_config(config)
  
    def __load_config(self, config):
        # 
#         raw_input_cfg = os.path.abspath(config.get_opt(Config.OPT_RAW_DATA_PATH))
        raw_input_cfg = os.path.join(os.getcwd(), config.get_opt(Config.OPT_RAW_DATA_PATH))
        
        if not os.path.exists(raw_input_cfg) or not os.path.isdir(raw_input_cfg):
            logger.exception("raw data directory does not exist: %s.", raw_input_cfg)
            exit(1)
        for ll in os.listdir(raw_input_cfg):
#            if os.path.isfile(ll) and ll.endswith('.data'):
            if ll.endswith('.data'):                
                self._raw_input_files.append(os.path.join(raw_input_cfg, ll))              
        logger.debug('configured raw data input list: %s', self._raw_input_files)
        if len(self._raw_input_files) == 0:
            logger.exception('cannot read any raw input data, pwdcore exit.')
            exit(1)
        #
        try:
#             kw_fd = open(config.get_opt(Config.OPT_KW_DICT), 'r')
            kw_fd = open(os.path.join(os.getcwd(), config.get_opt(Config.OPT_KW_DICT)), 'r')
            #
            self.__extract_kw(kw_fd)
        except Exception, e:
            logger.exception(e)
            logger.fatal("cannot extract PWP title key words %s, pwdcore exit.", \
                         os.path.join(os.getcwd(), config.get_opt(Config.OPT_KW_DICT)))
            exit(1)
        finally:
            kw_fd.close()
        #
        try:
#             pwsd_fd = open(config.get_opt(Config.OPT_PW_RES_DICT), 'r')
            pwsd_fd = open(os.path.join(os.getcwd(), config.get_opt(Config.OPT_PW_RES_DICT)), 'r')
            self.__extract_pwsd(pwsd_fd)
        except Exception, e:
            logger.exception(e)
            logger.fatal("cannot open %s, pwdcore exit.", \
                         os.path.join(os.getcwd(), config.get_opt(Config.OPT_PW_RES_DICT)))
            exit(1)
        finally:
            pwsd_fd.close()
        #
        try:
#             fname = config.get_opt(Config.OPT_OUTPUT) + '/' + time.strftime('%Y-%m-%d_', time.localtime()) + 'pwpd.out'
            fname = os.path.join(os.getcwd(), config.get_opt(Config.OPT_OUTPUT), time.strftime('%Y-%m-%d_', time.localtime()) + 'pwpd.out')
            self._out_fname = fname
        except Exception, e:
            logger.exception(e)
            logger.fatal("cannot open %s, pwdcore exit.", os.path.join(os.getcwd(), config.get_opt(Config.OPT_OUTPUT)))
            exit(1)
    
    def __extract_kw(self, fd):
        for line in fd:
            #@todo: change kw_dict encoding configureable
            if line.strip():
                self._kw_ll.append(line.strip().decode('utf-8'))
        logger.info('extract PWP title key words completed, number: %d', len(self._kw_ll))
        return len(self._kw_ll)
    
    def __extract_pwsd(self, fd):
        entry_num = 0
        for i in range(PhishingPageResLocator.MAX_DIR_STEP):
            self._pwsd_level[i+1] = list()
#        logger.debug('PhishingPageFeature extract pwsd.')
        
        for line in fd:
            if not line.startswith('#') and line.strip():
                ret = line.strip().split('\t')
                if not len(ret) == 3:
                    continue
                # error config. step, drop the entry
                if int(ret[1]) > PhishingPageResLocator.MAX_DIR_STEP:
                    continue
                #
                pprl = PhishingPageResLocator(ret[0], ret[1], ret[2])
                self._pwsd_level[pprl.step].append(pprl)
                entry_num += 1
#                self._pwsd_ll.append(PhishingPageResLocator(ret[0], int(ret[1]), bool(ret[2])))

#        logger.info('extract pwsd completed, number: %d', len(self._pwsd_ll))
#        return len(self._pwsd_ll)
        logger.info('extract PWP resource directory pattern completed, number: %d', entry_num)
        return entry_num
    
    def get_ppkw_list(self):
        return self._kw_ll
    
    def get_pwsd_list(self):
        return self._pwsd_ll
    
    def get_pwsd_level(self):
        return self._pwsd_level
    
    def get_raw_input_files(self):
        return self._raw_input_files
#
class PhishingPageResLocator(object):
    MAX_DIR_STEP = 3
    
    def __init__(self, dirt, step, is_end):
        self.dirt = dirt
        self.step = int(step)
        self.is_end = int(is_end)

class PWPDRunner(object):
    
    def __init__(self, config, timeout, url):
        self.cfg = config
        self.timeout = timeout
        self.url = url
        
        self._init_core(config, timeout, url)
        
    def _init_core(self, config, timeout, url):
        
        # init. PhishingPageFeature & PhishingPageDetector
        self.ppfeature = PhishingPageFeature(config)
        self.ppdetector = PhishingPageDetector(config, self.ppfeature, timeout)
        logger.info('init. ppfeature & ppdetector done.')
        #

    def detecting_start(self):
        out_fname = os.path.join(os.getcwd(), self.cfg.get_opt(Config.OPT_OUTPUT), time.strftime('%Y-%m-%d_', time.localtime()) + 'pwpd.out')
        try:
            out_fd = open(out_fname, 'w')
        except Exception, e:
            logger.exception(e)
            logger.fatal('cannot open output file: %s, PWPDRunner exit.', out_fname)
            exit(1)
        #
        pw_stat = 0
        
        # try to construct all suspected urls. 
        pwsd_level = self.ppfeature.get_pwsd_level().items()
        pwsd_level.sort(cmp = lambda x, y : cmp(x[0], y[0]))
        pwsd_ll = self.__construct_path(pwsd_level)
        logger.info('phishing resource path constructed: %d', len(pwsd_ll))
    
        # =========================================
        raw_input_files = self.ppfeature.get_raw_input_files()
        for fname in raw_input_files:
            try:
                input_fd = open(fname, 'r')
            except Exception, e:
                logger.exception(e)
                logger.fatal('failed to open raw input file: %s, cmdmain exit', fname)
                exit(1)
            #
            dnames = 0
            for line in input_fd:
                dname = line.strip().split('\t')[0]           
                dnames += self.__push_req_task(dname, pwsd_ll)
            # url push end
            input_fd.close()  
            logger.info('FILE [%s] total target URLs: %d', fname, dnames)
        #
        ll = 1
        while self.ppdetector.left_task():
            ll += 1
            if ll > 100:
                ll = 1
                print >> sys.stdout, 'residual detecting URLs ==>>  %d' %self.ppdetector.left_reqTask()
                logger.info('residual detecting URLs ==>>  %d', self.ppdetector.left_reqTask())
            #
            item = self.ppdetector.pop_pw()
            if item:
                req, content = item
                print >> out_fd, '[%s]\t%s' %(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), req)
                pw_stat += 1
            #
            else:
                time.sleep(0.1)
        # END while
        out_fd.close()
        logger.info("total phishing pages detected: %d", pw_stat)
    
        logger.info('############### Phishing web page detecting finished. ################## \n')
        # =========================================
    
    def __push_req_task(self, dname, pwsd_ll):
        self.ppdetector.push_task('http://' + dname)
        url_cont = 1
        #
        for ll in pwsd_ll:
            url = 'http://' + dname + '/' + ll
            self.ppdetector.push_task(url)
            url_cont +=1 
        #
        return url_cont        
    ##
    # @param pwsd_level: list object from dict.items(); (key, val) = (step, list_object)
    # 
    def __construct_path(self, pwsd_level):
        pwsd_ll = list()
        #
        for pprl in pwsd_level[0][1]:
            #
            pwsd_ll.append(pprl.dirt)
            #
            if not pprl.is_end:
                ll = self.__construct_path(pwsd_level[1:])
                for lll in ll:
                    pwsd_ll.append(pprl.dirt + '/' + lll)
        
        return pwsd_ll
    
    # abolished func.
    def __construct_path_main(self, dirt, level, pwsd_level):
        if PhishingPageResLocator.MAX_DIR_STEP == level + 1:
            pass
        
        ret = dirt + '/'
        
        for pprl in pwsd_level[0][1]:
            if pprl.is_end:
                ret += pprl.dirt
            else:
                ret += self.__construct_path_main()
            
            return ret
#
if __name__ == '__main__':
    DEFAULT_CONFIG = os.path.join(os.getcwd(), '../../etc/pwpd_cert.conf')
        